{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Adding RBF layer to prevent adversarial examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from keras.datasets import mnist\n",
    "from keras.utils import np_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded MNIST test data.\n"
     ]
    }
   ],
   "source": [
    "# prepare MNIST data set \n",
    "\n",
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "print(\"Loaded MNIST test data.\")\n",
    "\n",
    "X_train = X_train.reshape(60000, 784)\n",
    "X_test = X_test.reshape(10000, 784)\n",
    "X_train = X_train.astype('float32')\n",
    "X_test = X_test.astype('float32')\n",
    "X_train /= 255\n",
    "X_test /= 255\n",
    "\n",
    "Y_train = np_utils.to_categorical(y_train, 10)\n",
    "Y_test = np_utils.to_categorical(y_test, 10)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluate MLP network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from eval_adversarial import eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Load model ... ok\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/petra/cleverhans/cleverhans/utils_th.py:171: UserWarning: CleverHans support for Theano is deprecated and will be dropped on 2017-11-08.\n",
      "  warnings.warn(\"CleverHans support for Theano is deprecated and \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy on legitimate test examples: 0.9855\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/petra/cleverhans/cleverhans/attacks_th.py:36: UserWarning: CleverHans support for Theano is deprecated and will be dropped on 2017-11-08.\n",
      "  warnings.warn(\"CleverHans support for Theano is deprecated and \"\n",
      "/home/petra/cleverhans/cleverhans/utils_th.py:76: UserWarning: CleverHans support for Theano is deprecated and will be dropped on 2017-11-08.\n",
      "  warnings.warn(\"CleverHans support for Theano is deprecated and \"\n",
      "/home/petra/cleverhans/cleverhans/utils_th.py:231: UserWarning: CleverHans support for Theano is deprecated and will be dropped on 2017-11-08.\n",
      "  warnings.warn(\"CleverHans support for Theano is deprecated and \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy on adversarial examples: 0.026\n"
     ]
    }
   ],
   "source": [
    "eval(\"mlp\", X_train, Y_train, X_test, Y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create new model by adding RBF layer "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from mnist_add_rbf import add_rbf_layer\n",
    "from keras.models import model_from_json\n",
    "\n",
    "# load original MLP\n",
    "with open(\"models/mlp.json\") as f:\n",
    "    mlp = model_from_json(f.read()) \n",
    "mlp.load_weights(\"models/mlp_weights.h5\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 512)               401920    \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 512)               262656    \n",
      "_________________________________________________________________\n",
      "activation_2 (Activation)    (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 10)                5130      \n",
      "_________________________________________________________________\n",
      "activation_3 (Activation)    (None, 10)                0         \n",
      "_________________________________________________________________\n",
      "rbf_layer_2 (RBFLayer)       (None, 300)               3300      \n",
      "_________________________________________________________________\n",
      "dense_rbf (Dense)            (None, 10)                3000      \n",
      "_________________________________________________________________\n",
      "Activation_rbf (Activation)  (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 676,006\n",
      "Trainable params: 676,006\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/3\n",
      "60000/60000 [==============================] - 19s - loss: 0.7812 - acc: 0.8531    \n",
      "Epoch 2/3\n",
      "60000/60000 [==============================] - 22s - loss: 0.0521 - acc: 0.9874    \n",
      "Epoch 3/3\n",
      "60000/60000 [==============================] - 28s - loss: 0.0419 - acc: 0.9897    \n",
      "Test Accuracy:  98.08\n"
     ]
    }
   ],
   "source": [
    "# create new model by adding RBF layer  \n",
    "mlp_rbf =  add_rbf_layer(mlp, 2.0, X_train, Y_train, X_test, Y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# save new model \n",
    "json_string = mlp_rbf.to_json()\n",
    "with open(\"models/mlprbf.json\",\"w\") as f:\n",
    "    f.write(json_string) \n",
    "mlp_rbf.save_weights(\"models/mlprbf_weights.h5\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Eval new model  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Load model ... ok\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/petra/cleverhans/cleverhans/utils_th.py:171: UserWarning: CleverHans support for Theano is deprecated and will be dropped on 2017-11-08.\n",
      "  warnings.warn(\"CleverHans support for Theano is deprecated and \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy on legitimate test examples: 0.9808\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/petra/cleverhans/cleverhans/attacks_th.py:36: UserWarning: CleverHans support for Theano is deprecated and will be dropped on 2017-11-08.\n",
      "  warnings.warn(\"CleverHans support for Theano is deprecated and \"\n",
      "/home/petra/cleverhans/cleverhans/utils_th.py:76: UserWarning: CleverHans support for Theano is deprecated and will be dropped on 2017-11-08.\n",
      "  warnings.warn(\"CleverHans support for Theano is deprecated and \"\n",
      "/home/petra/cleverhans/cleverhans/utils_th.py:231: UserWarning: CleverHans support for Theano is deprecated and will be dropped on 2017-11-08.\n",
      "  warnings.warn(\"CleverHans support for Theano is deprecated and \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy on adversarial examples: 0.745\n"
     ]
    }
   ],
   "source": [
    "# eval new model \n",
    "eval(\"mlprbf\", X_train, Y_train, X_test, Y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
